import mlflow
import pandas as pd
import mlflow
import pandas as pd
def generate_recommendations_table(experiment_ids, aggregation_function="common_features", note="sizes_acts", group_type="sim"):
all_rows = []
for exp_id in experiment_ids:
runs = mlflow.search_runs(
experiment_ids=[exp_id],
output_format="list"
)
for run in runs:
if run.data.params.get("note") != note or run.data.params.get("SAE_fusion_strategy") != aggregation_function or run.data.params.get("group_type") != group_type:
continue
dataset = run.data.params.get("dataset", f"Exp-{exp_id}")
dim = int(run.data.params.get("embedding_dim", 0))
topk = int(run.data.params.get("top_k", 0))
row_key = (dim, topk)
metrics = {
(dataset, "G/mean"): run.data.metrics.get("CommonItemsNDCG20/median"),
(dataset, "U/mean"): run.data.metrics.get("NDCG20/mean"),
(dataset, "U/min"): run.data.metrics.get("NDCG20/min"),
(dataset, "Pop"): run.data.metrics.get("Popularity/mean"),
}
all_rows.append((row_key, metrics))
# Build DataFrame from records
records = {}
for key, metrics in all_rows:
if key not in records:
records[key] = {}
records[key].update(metrics)
df = pd.DataFrame.from_dict(records, orient="index")
df.index.names = ["Dimensions", "TopK"]
# Sort and reindex columns by dataset then metric
df = df.sort_index(axis=1, level=[0, 1]).sort_values(
by=["Dimensions", "TopK"]
)
return df.reset_index()
def generate_sae_table(experiment_ids, note=None):
all_rows = []
for exp_id in experiment_ids:
runs = mlflow.search_runs(
experiment_ids=[exp_id],
output_format="list"
)
for run in runs:
if run.data.params.get("note") != note:
continue
dataset = run.data.params.get("dataset", f"Exp-{exp_id}")
dim = int(run.data.params.get("embedding_dim", 0))
base_factors = float(run.data.params.get("base_factors", 0.0))
topk = int(run.data.params.get("top_k", 0))
row_key = (dim, topk)
metrics = {
(dataset, "CS"): run.data.metrics.get("CosineSim/test"),
(dataset, "Deg"): run.data.metrics.get("NDCG20_Degradation/test"),
(dataset, "Deads"): run.data.metrics.get("DeadNeurons/test")
}
all_rows.append((row_key, metrics))
# Build DataFrame from records
records = {}
for key, metrics in all_rows:
if key not in records:
records[key] = {}
records[key].update(metrics)
df = pd.DataFrame.from_dict(records, orient="index")
df.index.names = ["Dimensions", "TopK"]
# Sort and reindex columns by dataset then metric
df = df.sort_index(axis=1, level=[0, 1]).sort_values(
by=["Dimensions", "TopK"]
)
return df.reset_index()
import mlflow
import pandas as pd
def generate_common_features_table(experiment_ids, note="sizes_acts"):
all_rows = []
for exp_id in experiment_ids:
runs = mlflow.search_runs(
experiment_ids=[exp_id],
filter_string=f"params.note = '{note}'",
output_format="list"
)
for run in runs:
params = run.data.params
metrics = run.data.metrics
if params.get("note") != note:
continue
dataset = params.get("dataset", f"Exp-{exp_id}")
group_type = params.get("group_type", "unknown").capitalize()
dim = int(params.get("embedding_dim", 0))
topk = int(params.get("top_k", 0))
value = metrics.get("common_features/mean", None)
if value is None:
continue
row_key = (dim, topk)
all_rows.append((row_key, (dataset, group_type), value))
# Create dictionary for DataFrame
records = {}
for row_key, col_key, value in all_rows:
if row_key not in records:
records[row_key] = {}
records[row_key][col_key] = value
df = pd.DataFrame.from_dict(records, orient="index")
df.index.names = ["Dimensions", "TopK"]
# Reorder columns
if not df.empty:
datasets = ['LastFM1k',"MovieLens"]
subcols = ["Sim", "Random", "Div"]
col_order = [(d, s) for d in datasets for s in subcols]
df = df.reindex(columns=pd.MultiIndex.from_tuples(col_order)).sort_values(
by=["Dimensions", "TopK"]
)
return df.reset_index()
Sizes and selection of k for TopKSAE¶
Table of reconstrictions metrics¶
- CS - Cosine Similarity of original and reconstructed embeddings
- Deads - Percentage of dead neurons in the sparse embedding
- Deg - Degradation of NDCG between ELSA model and ELSA + Autoencoder
sae_experiments = ['657713966175362303', '852893065079987597']
table = generate_sae_table(sae_experiments, note="sizes_L2")
table.to_latex(
"sae_table.tex",
index=False,
float_format="%.3f",
column_format="lccc|ccc|ccc",
escape=False,
caption="SAE Table",
label="tab:sae_table"
)
table
| Dimensions | TopK | LastFM1k | MovieLens | |||||
|---|---|---|---|---|---|---|---|---|
| CS | Deads | Deg | CS | Deads | Deg | |||
| 0 | 1024 | 32 | 0.902610 | 0.224609 | -0.013318 | 0.947174 | 0.000000 | -0.011906 |
| 1 | 1024 | 64 | 0.925567 | 0.083008 | -0.006580 | 0.969829 | 0.000000 | -0.005658 |
| 2 | 1024 | 128 | 0.948935 | 0.012695 | -0.007478 | 0.991409 | 0.000000 | -0.002273 |
| 3 | 2048 | 32 | 0.902435 | 0.532227 | -0.002024 | 0.942687 | 0.000000 | -0.014361 |
| 4 | 2048 | 64 | 0.924614 | 0.250977 | -0.015078 | 0.965788 | 0.000000 | -0.007023 |
| 5 | 2048 | 128 | 0.950432 | 0.096680 | -0.018785 | 0.986710 | 0.000000 | -0.002464 |
| 6 | 4096 | 32 | 0.897760 | 0.743164 | -0.002289 | 0.940955 | 0.006836 | -0.015302 |
| 7 | 4096 | 64 | 0.923694 | 0.539307 | 0.000231 | 0.960520 | 0.000000 | -0.009255 |
| 8 | 4096 | 128 | 0.949578 | 0.302979 | -0.008452 | 0.978029 | 0.000000 | -0.004489 |
Number of common activated neurons for different group types and datasets¶
- Sim - Similary groups
- Random - Random groups
- Div - Divergent groups
experiments = ['228719589483846826','962723054918039068']
generate_common_features_table(experiments, note="sizes_L2_with_acts")
| Dimensions | TopK | LastFM1k | MovieLens | |||||
|---|---|---|---|---|---|---|---|---|
| Sim | Random | Div | Sim | Random | Div | |||
| 0 | 1024 | 32 | 4.31 | 2.08 | 0.12 | 3.76 | 2.33 | 0.07 |
| 1 | 1024 | 64 | 9.19 | 4.35 | 0.51 | 8.84 | 6.07 | 1.28 |
| 2 | 1024 | 128 | 24.02 | 13.02 | 2.66 | 18.21 | 14.42 | 8.22 |
| 3 | 2048 | 32 | 3.30 | 1.10 | 0.01 | 2.88 | 1.95 | 0.05 |
| 4 | 2048 | 64 | 6.99 | 2.96 | 0.16 | 7.09 | 4.86 | 0.54 |
| 5 | 2048 | 128 | 18.56 | 9.20 | 1.16 | 17.18 | 13.47 | 5.54 |
| 6 | 4096 | 32 | 2.97 | 0.87 | 0.00 | 2.12 | 1.41 | 0.02 |
| 7 | 4096 | 64 | 6.27 | 2.54 | 0.11 | 4.10 | 2.81 | 0.20 |
| 8 | 4096 | 128 | 15.96 | 7.24 | 0.78 | 9.68 | 6.63 | 1.15 |
SAE group recommendation performance for Common Feature aggregation function and similar groups¶
- G/mean - averaged NDCG@20, where positive interactions are the ones that are shared by all group members
- Pop - Averaged popularity of recommended items, where popularity of item is defined as the number of users that interacted with it divided by the number of users that interacted with the most popular item in the dataset
- U/mean - averaged individual mean NDCG@20 across all group members
- U/min - averaged individual minimum NDCG@20 across all group members
experiment_ids = ['333391697323445885', '523100174176986081']
# nejdrive common_features
table = generate_recommendations_table(experiment_ids, aggregation_function="common_features", note="sizes_L2_with_acts")
table
| Dimensions | TopK | LastFM1k | MovieLens | |||||||
|---|---|---|---|---|---|---|---|---|---|---|
| G/mean | Pop | U/mean | U/min | G/mean | Pop | U/mean | U/min | |||
| 0 | 1024 | 32 | 0.516076 | 0.661761 | 0.710516 | 0.474797 | 0.505553 | 0.479698 | 0.573466 | 0.410169 |
| 1 | 1024 | 64 | 0.476248 | 0.622218 | 0.725610 | 0.503415 | 0.224323 | 0.460364 | 0.617116 | 0.478394 |
| 2 | 1024 | 128 | 0.653500 | 0.590282 | 0.752573 | 0.578581 | 0.446518 | 0.457800 | 0.635899 | 0.476318 |
| 3 | 2048 | 32 | 0.475637 | 0.532887 | 0.711401 | 0.497113 | 0.560082 | 0.464062 | 0.565908 | 0.422937 |
| 4 | 2048 | 64 | 0.499871 | 0.623627 | 0.736160 | 0.518922 | 0.490893 | 0.399112 | 0.570854 | 0.439375 |
| 5 | 2048 | 128 | 0.618539 | 0.604613 | 0.767662 | 0.581601 | 0.383647 | 0.445989 | 0.627027 | 0.485645 |
| 6 | 4096 | 32 | 0.466293 | 0.583380 | 0.712037 | 0.450781 | 0.696985 | 0.462816 | 0.555817 | 0.384180 |
| 7 | 4096 | 64 | 0.617026 | 0.608345 | 0.720465 | 0.499116 | 0.628033 | 0.528345 | 0.601176 | 0.426782 |
| 8 | 4096 | 128 | 0.654298 | 0.622641 | 0.750652 | 0.546807 | 0.573619 | 0.458854 | 0.594654 | 0.467905 |
SAE group recommendation performance for Average aggregation function and similar groups¶
table = generate_recommendations_table(experiment_ids, aggregation_function="average", note="sizes_L2_with_acts")
averaged_values = table.mean(axis=0, numeric_only=True)
table
| Dimensions | TopK | LastFM1k | MovieLens | |||||||
|---|---|---|---|---|---|---|---|---|---|---|
| G/mean | Pop | U/mean | U/min | G/mean | Pop | U/mean | U/min | |||
| 0 | 1024 | 32 | 0.519204 | 0.600704 | 0.809246 | 0.635029 | 0.590178 | 0.531097 | 0.686556 | 0.578793 |
| 1 | 1024 | 64 | 0.559005 | 0.611901 | 0.812148 | 0.647854 | 0.744141 | 0.537555 | 0.696120 | 0.576863 |
| 2 | 1024 | 128 | 0.572528 | 0.608556 | 0.811758 | 0.648562 | 0.663719 | 0.533581 | 0.690955 | 0.575475 |
| 3 | 2048 | 32 | 0.582310 | 0.617113 | 0.817844 | 0.637349 | 0.656759 | 0.540976 | 0.686509 | 0.582331 |
| 4 | 2048 | 64 | 0.575728 | 0.610493 | 0.824395 | 0.665815 | 0.666919 | 0.533732 | 0.692607 | 0.577328 |
| 5 | 2048 | 128 | 0.645750 | 0.598873 | 0.816464 | 0.667066 | 0.703290 | 0.527509 | 0.691604 | 0.574714 |
| 6 | 4096 | 32 | 0.586220 | 0.614859 | 0.813526 | 0.651378 | 0.572665 | 0.544739 | 0.680556 | 0.564184 |
| 7 | 4096 | 64 | 0.584140 | 0.610422 | 0.819364 | 0.649836 | 0.692339 | 0.540435 | 0.685338 | 0.586880 |
| 8 | 4096 | 128 | 0.617503 | 0.613028 | 0.818175 | 0.641317 | 0.684063 | 0.532440 | 0.692533 | 0.583607 |
SAE group recommendation performance for Average aggregation function and random groups¶
generate_recommendations_table(experiment_ids, aggregation_function="common_features", note="sizes_L2_with_acts", group_type="random").round(3)
| Dimensions | TopK | LastFM1k | MovieLens | |||||||
|---|---|---|---|---|---|---|---|---|---|---|
| G/mean | Pop | U/mean | U/min | G/mean | Pop | U/mean | U/min | |||
| 0 | 1024 | 32 | 0.245 | 0.536 | 0.547 | 0.271 | 0.175 | 0.488 | 0.514 | 0.373 |
| 1 | 1024 | 64 | 0.173 | 0.596 | 0.602 | 0.363 | 0.673 | 0.441 | 0.544 | 0.384 |
| 2 | 1024 | 128 | 0.250 | 0.600 | 0.642 | 0.423 | 0.662 | 0.456 | 0.589 | 0.420 |
| 3 | 2048 | 32 | 0.091 | 0.146 | 0.209 | 0.071 | 0.299 | 0.435 | 0.473 | 0.282 |
| 4 | 2048 | 64 | 0.197 | 0.538 | 0.517 | 0.308 | 0.266 | 0.412 | 0.510 | 0.344 |
| 5 | 2048 | 128 | 0.268 | 0.585 | 0.618 | 0.398 | 0.501 | 0.449 | 0.592 | 0.420 |
| 6 | 4096 | 32 | 0.000 | 0.130 | 0.150 | 0.037 | 0.152 | 0.424 | 0.466 | 0.275 |
| 7 | 4096 | 64 | 0.399 | 0.484 | 0.442 | 0.208 | 0.531 | 0.464 | 0.546 | 0.404 |
| 8 | 4096 | 128 | 0.377 | 0.581 | 0.577 | 0.345 | 0.499 | 0.483 | 0.585 | 0.412 |
SAE group recommendation performance for Average aggregation function and divergent groups¶
generate_recommendations_table(experiment_ids, aggregation_function="average", note="sizes_L2_with_acts", group_type="div")
| Dimensions | TopK | LastFM1k | MovieLens | |||||||
|---|---|---|---|---|---|---|---|---|---|---|
| G/mean | Pop | U/mean | U/min | G/mean | Pop | U/mean | U/min | |||
| 0 | 1024 | 32 | 0.502890 | 0.633486 | 0.672541 | 0.462534 | 0.151301 | 0.426824 | 0.620246 | 0.447686 |
| 1 | 1024 | 64 | 0.464753 | 0.607923 | 0.660175 | 0.442029 | 0.130324 | 0.420144 | 0.624392 | 0.464793 |
| 2 | 1024 | 128 | 0.412558 | 0.600951 | 0.660433 | 0.419930 | 0.117063 | 0.422243 | 0.629856 | 0.435937 |
| 3 | 2048 | 32 | 0.434981 | 0.617218 | 0.679557 | 0.481588 | 0.151301 | 0.435773 | 0.626579 | 0.428157 |
| 4 | 2048 | 64 | 0.445476 | 0.611796 | 0.669399 | 0.457770 | 0.117063 | 0.431221 | 0.623958 | 0.440388 |
| 5 | 2048 | 128 | 0.467075 | 0.608979 | 0.658732 | 0.441154 | 0.130324 | 0.397205 | 0.607251 | 0.416461 |
| 6 | 4096 | 32 | 0.515982 | 0.615845 | 0.679850 | 0.469024 | 0.107789 | 0.445748 | 0.636909 | 0.472828 |
| 7 | 4096 | 64 | 0.465149 | 0.610422 | 0.679596 | 0.463612 | 0.100867 | 0.445096 | 0.632272 | 0.460025 |
| 8 | 4096 | 128 | 0.447571 | 0.609472 | 0.663374 | 0.445680 | 0.130324 | 0.400512 | 0.623498 | 0.417764 |